package crawler.proxy.uitl;

import com.mongodb.client.MongoCollection;
import mongo.DateUtil;
import mongo.MongoTest;
import net.sf.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Pattern;

/**
 * 代理IP获取
 * 1 获取代理IP列表，
 * 2 验证可用性(不可用IP过多时停止抓取)
 * 3 保存至redis
 * USER: lintc 【lintiancong@zhuojianchina.com】
 * DATE: 2017-09-01 15:53
 */
public class ProxyIp {
    public static void main(String[] args) throws IOException {
        MongoCollection<org.bson.Document> collection = MongoTest.test.getCollection("proxy_ip");
        String url = "http://www.66ip.cn/index.html";
        List<JSONObject> list = new ArrayList<>();
        JSONObject obj;
        for (int i = 1; i == 1 || list.size() != 0; i++) {
            String ipUrl = url.replace("index", i + "");
            list = getProxyIp(ipUrl);
            for (JSONObject ip : list) {
                obj = new JSONObject();
                obj.put("ip", ip.optString("ip"));
                obj.put("port", ip.optString("port"));
                obj.put("is_usable", true);
                obj.put("catch_url", ipUrl);
                obj.put("catch_date", DateUtil.getyyyy_MM_dd(new Date()));
                MongoTest.insertOne(collection, obj);
            }
        }
    }

    /**
     * 抓取IP
     * @param url
     * @return
     */
    public static List<JSONObject> getProxyIp(String url) {
        List<JSONObject> list = new ArrayList<>();
        Document doc;
        try {
            doc = Jsoup.connect(url).get();
            System.out.println(url);
            Elements es = doc.select("tr");
            es.forEach(element -> {
                for (Element e : element.children()) {
                    if (isIp(e.text())) {
                        JSONObject obj = new JSONObject();
                        obj.put("ip", e.text());
                        obj.put("port", e.nextElementSibling().text());
                        list.add(obj);
                        System.out.println(e.text() + ":" + e.nextElementSibling().text());
                        break;
                    }
                }
            });
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }

    /**
     * 验证IP格式
     * @param str
     * @return
     */
    public static boolean isIp(String str) {
        String pattern = "(\\d{0,3}\\.){3}\\d{0,3}";
        if (Pattern.matches(pattern, str.trim())) {
            return true;
        }
        return false;
    }
}
